{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 使用selenium爬取，因为该网页是动态点击的，需要模拟点击事件，做之前如果先检查Chrome浏览器和driver是否版本一致，版本报错参考https://blog.csdn.net/qq_40604853/article/details/81388078 还有外网比较慢，可以不用先爬五年的，页数太多"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "import time\n",
    "import json\n",
    "import pandas as pd\n",
    "import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-32-d1a954f1a6eb>:5: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
      "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n",
      "  for i in tqdm.tqdm_notebook(range(71)):\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4e9fcaa0c33f4fe3ba063d35228e6c05",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=71.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "url=\"https://www.nasdaq.com/market-activity/stocks/amzn/historical\"\n",
    "driver = webdriver.Chrome()\n",
    "#3、driver发生get请求\n",
    "driver.get(url)\n",
    "for i in tqdm.tqdm_notebook(range(71)):\n",
    "    if i==0:        \n",
    "        #第一页\n",
    "        #进入五年模式,找到5年这个按钮对应的源代码，将xpath复制下来（可以问我）\n",
    "        element=driver.find_element_by_xpath('/html/body/div[2]/div/main/div[2]/div[4]/div[3]/div/div[1]/div/div[1]/div[3]/div/div/div/button[5]')\n",
    "        driver.execute_script(\"arguments[0].click();\", element)\n",
    "        #外网加载速度比较慢，手动阻塞，以免响应不及时\n",
    "        time.sleep(10)\n",
    "        #保存网页源代码\n",
    "        data = driver.page_source\n",
    "        a=pd.read_html(data)#提取表格\n",
    "        df_stocks=pd.DataFrame(a[0])#变为数据框\n",
    "    else:\n",
    "        #第二页以后需要点击下一页，同样将下一页的xpath复制下来\n",
    "        element=driver.find_element_by_xpath('/html/body/div[2]/div/main/div[2]/div[4]/div[3]/div/div[1]/div/div[1]/div[5]/button[2]')\n",
    "        driver.execute_script(\"arguments[0].click();\", element)\n",
    "        #外网加载速度比较慢，手动阻塞，以免响应不及时\n",
    "        time.sleep(5)\n",
    "        #保存网页源代码\n",
    "        data = driver.page_source\n",
    "        a=pd.read_html(data)\n",
    "        df_stocks=pd.concat([df_stocks,pd.DataFrame(a[0])])#将多页的数据拼接到一个数据框里面"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<function str.join(iterable, /)>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "/html/body/div[2]/div/main/div[2]/div[4]/div[3]/div/div[1]/div/div[1]/div[5]/button[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>Close/Last</th>\n",
       "      <th>Volume</th>\n",
       "      <th>Open</th>\n",
       "      <th>High</th>\n",
       "      <th>Low</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>12/31/2021</td>\n",
       "      <td>$3334.34</td>\n",
       "      <td>2386287</td>\n",
       "      <td>$3379.12</td>\n",
       "      <td>$3387</td>\n",
       "      <td>$3331.17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>12/30/2021</td>\n",
       "      <td>$3372.89</td>\n",
       "      <td>1879155</td>\n",
       "      <td>$3394</td>\n",
       "      <td>$3417.76</td>\n",
       "      <td>$3370.48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>12/29/2021</td>\n",
       "      <td>$3384.02</td>\n",
       "      <td>1787702</td>\n",
       "      <td>$3416.8</td>\n",
       "      <td>$3424.24</td>\n",
       "      <td>$3372.0101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>12/28/2021</td>\n",
       "      <td>$3413.22</td>\n",
       "      <td>2735593</td>\n",
       "      <td>$3403.65</td>\n",
       "      <td>$3443.5176</td>\n",
       "      <td>$3382.71</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>12/27/2021</td>\n",
       "      <td>$3393.39</td>\n",
       "      <td>2942341</td>\n",
       "      <td>$3420.74</td>\n",
       "      <td>$3458.86</td>\n",
       "      <td>$3384.31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>12/23/2021</td>\n",
       "      <td>$3421.37</td>\n",
       "      <td>1839412</td>\n",
       "      <td>$3408.56</td>\n",
       "      <td>$3439.5</td>\n",
       "      <td>$3403</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>12/22/2021</td>\n",
       "      <td>$3420.74</td>\n",
       "      <td>2751773</td>\n",
       "      <td>$3385.395</td>\n",
       "      <td>$3441</td>\n",
       "      <td>$3370.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>12/21/2021</td>\n",
       "      <td>$3408.34</td>\n",
       "      <td>2797791</td>\n",
       "      <td>$3357.01</td>\n",
       "      <td>$3414.3299</td>\n",
       "      <td>$3312.95</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>12/20/2021</td>\n",
       "      <td>$3341.58</td>\n",
       "      <td>2868633</td>\n",
       "      <td>$3337</td>\n",
       "      <td>$3357.4851</td>\n",
       "      <td>$3312</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>12/17/2021</td>\n",
       "      <td>$3400.35</td>\n",
       "      <td>4287758</td>\n",
       "      <td>$3354.21</td>\n",
       "      <td>$3417.97</td>\n",
       "      <td>$3312.271</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>12/16/2021</td>\n",
       "      <td>$3377.42</td>\n",
       "      <td>3043842</td>\n",
       "      <td>$3467.371</td>\n",
       "      <td>$3483.32</td>\n",
       "      <td>$3363.205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>12/15/2021</td>\n",
       "      <td>$3466.3</td>\n",
       "      <td>3789713</td>\n",
       "      <td>$3371.96</td>\n",
       "      <td>$3472</td>\n",
       "      <td>$3303.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>12/14/2021</td>\n",
       "      <td>$3381.83</td>\n",
       "      <td>2798764</td>\n",
       "      <td>$3351</td>\n",
       "      <td>$3389.98</td>\n",
       "      <td>$3328.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>12/13/2021</td>\n",
       "      <td>$3391.35</td>\n",
       "      <td>3108450</td>\n",
       "      <td>$3440</td>\n",
       "      <td>$3442</td>\n",
       "      <td>$3382.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>12/10/2021</td>\n",
       "      <td>$3444.24</td>\n",
       "      <td>3034488</td>\n",
       "      <td>$3508.34</td>\n",
       "      <td>$3518.54</td>\n",
       "      <td>$3410</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>12/09/2021</td>\n",
       "      <td>$3483.42</td>\n",
       "      <td>2303091</td>\n",
       "      <td>$3515</td>\n",
       "      <td>$3539.39</td>\n",
       "      <td>$3482.79</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>12/08/2021</td>\n",
       "      <td>$3523.16</td>\n",
       "      <td>2262683</td>\n",
       "      <td>$3523.01</td>\n",
       "      <td>$3543.6</td>\n",
       "      <td>$3495.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>12/07/2021</td>\n",
       "      <td>$3523.29</td>\n",
       "      <td>3320536</td>\n",
       "      <td>$3492</td>\n",
       "      <td>$3549.99</td>\n",
       "      <td>$3466.69</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          Date Close/Last   Volume       Open        High         Low\n",
       "0   12/31/2021   $3334.34  2386287   $3379.12       $3387    $3331.17\n",
       "1   12/30/2021   $3372.89  1879155      $3394    $3417.76    $3370.48\n",
       "2   12/29/2021   $3384.02  1787702    $3416.8    $3424.24  $3372.0101\n",
       "3   12/28/2021   $3413.22  2735593   $3403.65  $3443.5176    $3382.71\n",
       "4   12/27/2021   $3393.39  2942341   $3420.74    $3458.86    $3384.31\n",
       "5   12/23/2021   $3421.37  1839412   $3408.56     $3439.5       $3403\n",
       "6   12/22/2021   $3420.74  2751773  $3385.395       $3441    $3370.01\n",
       "7   12/21/2021   $3408.34  2797791   $3357.01  $3414.3299    $3312.95\n",
       "8   12/20/2021   $3341.58  2868633      $3337  $3357.4851       $3312\n",
       "9   12/17/2021   $3400.35  4287758   $3354.21    $3417.97   $3312.271\n",
       "10  12/16/2021   $3377.42  3043842  $3467.371    $3483.32   $3363.205\n",
       "11  12/15/2021    $3466.3  3789713   $3371.96       $3472     $3303.9\n",
       "12  12/14/2021   $3381.83  2798764      $3351    $3389.98     $3328.8\n",
       "13  12/13/2021   $3391.35  3108450      $3440       $3442     $3382.6\n",
       "14  12/10/2021   $3444.24  3034488   $3508.34    $3518.54       $3410\n",
       "15  12/09/2021   $3483.42  2303091      $3515    $3539.39    $3482.79\n",
       "16  12/08/2021   $3523.16  2262683   $3523.01     $3543.6    $3495.01\n",
       "17  12/07/2021   $3523.29  3320536      $3492    $3549.99    $3466.69"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a=pd.read_html(data)\n",
    "pd.DataFrame(a[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 怎么爬取其他时间段的数据？\n",
    "### 怎么爬取其他股票的数据？\n",
    "### 怎么将数据框中的美元符号去掉？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>Close/Last</th>\n",
       "      <th>Volume</th>\n",
       "      <th>Open</th>\n",
       "      <th>High</th>\n",
       "      <th>Low</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>12/31/2021</td>\n",
       "      <td>$3334.34</td>\n",
       "      <td>2386287</td>\n",
       "      <td>$3379.12</td>\n",
       "      <td>$3387</td>\n",
       "      <td>$3331.17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>12/30/2021</td>\n",
       "      <td>$3372.89</td>\n",
       "      <td>1879155</td>\n",
       "      <td>$3394</td>\n",
       "      <td>$3417.76</td>\n",
       "      <td>$3370.48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>12/29/2021</td>\n",
       "      <td>$3384.02</td>\n",
       "      <td>1787702</td>\n",
       "      <td>$3416.8</td>\n",
       "      <td>$3424.24</td>\n",
       "      <td>$3372.0101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>12/28/2021</td>\n",
       "      <td>$3413.22</td>\n",
       "      <td>2735593</td>\n",
       "      <td>$3403.65</td>\n",
       "      <td>$3443.5176</td>\n",
       "      <td>$3382.71</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>12/27/2021</td>\n",
       "      <td>$3393.39</td>\n",
       "      <td>2942341</td>\n",
       "      <td>$3420.74</td>\n",
       "      <td>$3458.86</td>\n",
       "      <td>$3384.31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>01/09/2017</td>\n",
       "      <td>$796.92</td>\n",
       "      <td>3442864</td>\n",
       "      <td>$798</td>\n",
       "      <td>$801.7742</td>\n",
       "      <td>$791.77</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>01/06/2017</td>\n",
       "      <td>$795.99</td>\n",
       "      <td>5981169</td>\n",
       "      <td>$782.36</td>\n",
       "      <td>$799.44</td>\n",
       "      <td>$778.48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>01/05/2017</td>\n",
       "      <td>$780.45</td>\n",
       "      <td>5820195</td>\n",
       "      <td>$761.55</td>\n",
       "      <td>$782.3999</td>\n",
       "      <td>$760.2557</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>01/04/2017</td>\n",
       "      <td>$757.18</td>\n",
       "      <td>2505296</td>\n",
       "      <td>$758.39</td>\n",
       "      <td>$759.68</td>\n",
       "      <td>$754.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>01/03/2017</td>\n",
       "      <td>$753.67</td>\n",
       "      <td>3519877</td>\n",
       "      <td>$757.92</td>\n",
       "      <td>$758.7595</td>\n",
       "      <td>$747.7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1276 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          Date Close/Last   Volume      Open        High         Low\n",
       "0   12/31/2021   $3334.34  2386287  $3379.12       $3387    $3331.17\n",
       "1   12/30/2021   $3372.89  1879155     $3394    $3417.76    $3370.48\n",
       "2   12/29/2021   $3384.02  1787702   $3416.8    $3424.24  $3372.0101\n",
       "3   12/28/2021   $3413.22  2735593  $3403.65  $3443.5176    $3382.71\n",
       "4   12/27/2021   $3393.39  2942341  $3420.74    $3458.86    $3384.31\n",
       "..         ...        ...      ...       ...         ...         ...\n",
       "12  01/09/2017    $796.92  3442864      $798   $801.7742     $791.77\n",
       "13  01/06/2017    $795.99  5981169   $782.36     $799.44     $778.48\n",
       "14  01/05/2017    $780.45  5820195   $761.55   $782.3999   $760.2557\n",
       "15  01/04/2017    $757.18  2505296   $758.39     $759.68      $754.2\n",
       "16  01/03/2017    $753.67  3519877   $757.92   $758.7595      $747.7\n",
       "\n",
       "[1276 rows x 6 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_stocks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
